/*  This file is part of o.O.

    o.O is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    o.O is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with o.O.  If not, see <http://www.gnu.org/licenses/>.

    Copyright 2013 Oliver Katz */

/* HEY, MAN!

If you're trying to use o.O's tokenizer, this is the place to be. The
easiest way to make a new tokenizer is:
Tokenizer newTokenizer = Tokenizer(" \t\n", "\"", "( ) +");

Let me explain: the first argument is a string containing all the
whitespace characters you want to use. These will deliminate a string
like such:
'hello, world' -> 'hello,' 'world' (with whitespace: " ")

The second argument is a string containing all the quotation
characters. These mark the beginning and the end of a quotation. For
example:
'"hello, world"' -> '"hello, world"' (with whitespace: " " and
	quotes: "\"")

The last argument is a string containing a list of all the symbols
separated by whitespace characters. For example:
'i++' -> 'i' '++' (with symbols: "+ ++")
'i+1' -> 'i' '+' '1' (with the same symbols)

To actually tokenize, just do:
vector<Token> tokens = newTokenizer.tokenize("this string will be tokenized");

You can get the string contents of a Token with:
string tokenContents = tokens[0].get(); */

#ifndef __OO_PARSING_TOKENIZER_H
#define __OO_PARSING_TOKENIZER_H

#include "types.h"

namespace oO
{
	class Tokenizer
	{
	private:
		std::vector<std::string> symbols;
		std::string whitespace;
		std::string quotes;

		void sortSymbols();
		bool debug;

	public:
		Tokenizer() : debug(false) {}
		Tokenizer(std::string w) : debug(false), whitespace(w) {}
		Tokenizer(std::string w, std::string q) : debug(false), whitespace(w), quotes(q) {}
		Tokenizer(std::string w, std::string q, std::vector<std::string> s) : debug(false), whitespace(w), quotes(q), symbols(s) {sortSymbols();}
		Tokenizer(std::string w, std::string q, std::string s);
		void enableDebugging();
		std::vector<Token> tokenize(std::string s);
	};
}

#endif
